//
// Created by 11510 on 2024/7/29.
//

#ifndef WEB_DEDUPLICATION_PAGELIBPREPROCESSOR_H
#define WEB_DEDUPLICATION_PAGELIBPREPROCESSOR_H
#include "tinyxml2.h"
#include "Configuration.h"
#include "WebPage.h"
#include "cppjieba/Jieba.hpp"
#include "SplitTool.h"
#include <string>
#include <vector>
#include <unordered_map>
#include <map>
#include <algorithm>

class PageLibPreprocessor {
private:

    Simhasher *simHasher;
    SplitTool *jieba;
    std::vector<WebPage> pageLib;/* 网页库对象 */
    std::map<int, std::pair<size_t, size_t>> offsetLib;   /* 网页偏移库 */

    /* 所有文章的词语, 和该词出现的文章数 */
    std::unordered_map<std::string, int> allPageWordMap;
    /* 倒排索引表 */
    std::unordered_map<std::string, std::set<std::pair<int, double>>> invertIndexTable;

public:
    PageLibPreprocessor(Simhasher *simHasher, SplitTool *jieba) :
            simHasher(simHasher),
            jieba(jieba) {}

    ~PageLibPreprocessor() = default;

    /**
     * 读取网页库
     * @param filepath
     */
    void readInfoFromFile(const char *filepath) {
        parseRss(filepath);
    }

    /**
     * 计算topK和哈希值
     */
    void doProcess() {
        for (auto &page: pageLib) {
            page.hashCodeOfContent(simHasher);
        }
    }

    /**
     * 去重 O(n2)
     */
    void cutRedundantPages() {
        for (auto it = pageLib.begin(); it != pageLib.end(); ++it) {
            for (auto innerIt = it + 1; innerIt != pageLib.end();) {
                if (*it == *innerIt) {
                    innerIt = pageLib.erase(innerIt);
                } else {
                    ++innerIt;
                }
            }
        }
    }

    /**
     * 建立倒排索引表
     */
    void buildInvertIndexTable() {
        /* 统计每篇文章的词和词频 */
        std::for_each(pageLib.begin(), pageLib.end(), [this](WebPage &page) {
            page.splitWordAndCntFrequency(jieba);
        });

        /* 统计所有词在对应的文章数量 */
        for(const auto& elem: pageLib){
            for(const auto& entry: elem.getWordsMap()){
                allPageWordMap[entry.first]++;
            }
        }

        /* 建表,外层引用不能const */
        for (auto &elem: pageLib) {
            double sumOfSquaresW = .0F;

            for (const auto &entry: elem.getWordsMap()) {
                int TF = entry.second;
                int N = pageLib.size();
                int DF = allPageWordMap[entry.first];

//                TODO BUG
                double w = weight(TF, N, DF);
                sumOfSquaresW += w * w;

                elem.getWordWeightMap()[entry.first] = w;
            }

            for (const auto &entry: elem.getWordWeightMap()) {
                invertIndexTable[entry.first].insert({elem.getDocId(), entry.second / sqrt(sumOfSquaresW)});
            }
        }
    }

    double weight(double TF, int N, double DF) {
        return TF * ::log(N / (DF + 1.0F));
    }


    /**
     * 保存网页库，偏移库，倒排索引表
     * @param webXML
     * @param webOft
     * @param ivtIdxTab
     */
    void storeOnDisk(const char *webXML,
                     const char *webOft,
                     const char *ivtIdxTab) {
        storeWebXML(webXML);
        createOffsetLib(webXML);
        storeOffset(webOft);
        storeInvertIndexTable(ivtIdxTab);

    }

private:
    /**
     * 解析xml文件到vector
     * @param filename
     */
    void parseRss(const char *filename) {
        tinyxml2::XMLDocument document;
        document.LoadFile(filename);

        for (tinyxml2::XMLElement *doc = document.FirstChildElement("doc"); doc; doc = doc->NextSiblingElement("doc")) {

            tinyxml2::XMLElement *docId = doc->FirstChildElement("docid");
            tinyxml2::XMLElement *title = doc->FirstChildElement("title");
            tinyxml2::XMLElement *link = doc->FirstChildElement("link");
            tinyxml2::XMLElement *content = doc->FirstChildElement("content");

            pageLib.push_back(std::move(WebPage(::atoi(docId->GetText()),
                                                title->GetText(),
                                                link->GetText(),
                                                content->GetText())));
        }

    }

    /**
     * 存储xml文件
     * @param filename
     */
    void storeWebXML(const char *filename) {
        tinyxml2::XMLDocument result;

        /* id已经乱了，需要重新排列 */
        int cntDocumentId = 0;

        for (auto it = pageLib.begin(); it != pageLib.end(); it++) {

            tinyxml2::XMLElement *doc = result.NewElement("doc");
            result.InsertEndChild(doc);
            insertChild(&result, doc, "docid", std::to_string(cntDocumentId).c_str());
            insertChild(&result, doc, "title", it->getDocTitle().c_str());
            insertChild(&result, doc, "link", it->getDocUrl().c_str());
            insertChild(&result, doc, "content", it->getDocContent().c_str());

            cntDocumentId++;
        }

        result.SaveFile(filename);
    }

    void insertChild(tinyxml2::XMLDocument *page, tinyxml2::XMLElement *parent, const char *elemName, const char *elemText) {
        tinyxml2::XMLElement *element = page->NewElement(elemName);
        tinyxml2::XMLText *elementText = page->NewText(elemText);
        element->InsertEndChild(elementText);
        parent->InsertEndChild(element);
    }

    /**
     * 创建偏移库
     * @param filename
     */
    void createOffsetLib(const char *filename) {
        std::ifstream file(filename);
        if (!file.is_open()) {
            std::cerr << "Failed to open file: " << filename << std::endl;
            return;
        }

        std::stringstream buffer;
        buffer << file.rdbuf();
        std::string content = buffer.str();
        file.close();

        size_t pos = 0;
        int docId = 0;

        while ((pos = content.find("<doc>", pos)) != std::string::npos) {
            size_t start = pos;
            size_t end = content.find("</doc>", start);
            if (end == std::string::npos) {
                std::cerr << "Malformed XML: No matching </doc> for <doc> at position " << start << std::endl;
                break;
            }
            end += 6; // Length of "</doc>"

            size_t docidPos = content.find("<docid>", start);
            size_t docidEnd = content.find("</docid>", docidPos);
            if (docidPos == std::string::npos || docidEnd == std::string::npos) {
                std::cerr << "Malformed XML: No matching <docid> for <doc> at position " << start << std::endl;
                break;
            }

            std::string docidStr = content.substr(docidPos + 7, docidEnd - docidPos - 7); // Length of "<docid>"
            docId = stoi(docidStr);

            offsetLib[docId] = {start, end - start};
            pos = end;
        }
    }

    /**
     * 保存偏移库
     * @param filepath
     */
    void storeOffset(const char *filepath) {
        std::ofstream ofs(filepath);
        if (!ofs) {
            std::cerr << "ofstream open file failed\n";
            return;
        }

        for (const auto &entry: offsetLib) {
            ofs << entry.first << " "
                << entry.second.first << " "
                << entry.second.second << "\n";
        }
        ofs.close();
    }

    /**
     * 保存倒排索引表
     * @param filepath
     */
    void storeInvertIndexTable(const char *filepath){
        // 打开文件以进行写操作
        std::ofstream outFile(filepath);

        if (!outFile.is_open()) {
            std::cerr << "Failed to open file: " << filepath << "\n";
            return;
        }

        // 遍历 unordered_map 并写入文件
        for (const auto& entry : invertIndexTable) {
            outFile << entry.first; // 写入键
            for (const auto& pair : entry.second) {
                outFile << " " << pair.first << " " << pair.second; // 写入 int 和 double
            }
            outFile << std::endl; // 换行
        }

        // 关闭文件
        outFile.close();
    }
};


#endif //WEB_DEDUPLICATION_PAGELIBPREPROCESSOR_H
